# Ofri Keidar Inbal Wiesel 302933981 305331878

from Parser import Parser
from UniLidstone import UniLidstone
from BiLidstone import BiLidstone
from Backoff import Backoff
from OutputMngr import OutputMngr

'''
Creates necessary objects for Backoff model training on train set, evalates it on test set
and writes results to output file
'''
class MainApp:
    
    def __init__(self, develFileName, outputFileName):
        
        self.trainHeader = "<TRAIN"; # prefix of train header line
        self.testHeader = "<TEST"; # prefix of test header line
        self.wordDelim = " "; # delimiter between words in data set files
        self.VOCAB_SIZE = 300000; # language vocabulary size
        self.minLambda = 0; # minimal lambda value for bigram Lidstone to be cehcked
        self.maxLambda = 0.02; # maximal lambda value for bigram Lidstone to be cehcked
        self.step = 0.0001; # increment step between lambda values to be checked
        self.initUniLambda = 0.1; # initial value for lambda of unigram Lidstone
        self.initBiLambda = 0.0001; # initial value for lambda of bigram Lidstone

        # names of generated files
        self.trainFileName = "train.txt";
        self.validFileName = "validation.txt";

        # create train and validation files
        self.parser = Parser(self.trainHeader, self.wordDelim);
        self.parser.splitDataSet(develFileName, self.trainFileName, self.validFileName, 0.9);
        
        # count articles in train set
        numArticles = self.parser.countArticles(self.trainFileName);
        
        # initialize unigram Lidstone object
        numUniEventsTrain = self.parser.countUniEvents(self.trainFileName);
        uniObsVoc = self.parser.genUniObsVoc(self.trainFileName);
        self.uniLidstone = UniLidstone(self.VOCAB_SIZE, uniObsVoc, numUniEventsTrain, self.initUniLambda, numArticles);
        
        # initialize bigram Lidstone object
        biObsVoc = self.parser.genBiObsVoc(self.trainFileName);
        self.biLidstone = BiLidstone(self.VOCAB_SIZE, uniObsVoc, biObsVoc, self.initBiLambda, numArticles);
        
        # initialize Backoff object
        condWords = self.parser.getCondWords(self.trainFileName);
        self.backoff = Backoff(self.uniLidstone, self.biLidstone, condWords, self.initUniLambda, self.initBiLambda);
        
        # initialize output manager
        self.outputMngr = OutputMngr(outputFileName);
        
    '''
    Trains Backoff on test set and evaluates it on test set
    '''
    def run(self, develFileName, testFileName, firstInputWord, secondInputWord, outputFileName):
        
        # initialize output file
        self.initOutputFile(develFileName, testFileName, firstInputWord, secondInputWord, outputFileName);
        
        # process train set
        self.procTrain(develFileName, firstInputWord, secondInputWord);
                
        # process test set
        self.procTest(testFileName);
        
        # write table
        self.writeResultsTable(firstInputWord);
        
        # close output file
        self.outputMngr.closeOutput();
        
    '''
    Writes initial data to output file
    '''
    def initOutputFile(self, develFileName, testFileName, firstInputWord, secondInputWord, outputFileName):
        
        # write students details
        self.outputMngr.writePersonalDetails(("Ofri Keidar", "Inbal Wiesel"), ("302933981", "305331878"));
        
        # write given data to output file
        self.outputMngr.writeLine(develFileName);
        self.outputMngr.writeLine(testFileName);
        self.outputMngr.writeLine(firstInputWord + " " + secondInputWord);
        self.outputMngr.writeLine(outputFileName);
        self.outputMngr.writeLine(self.VOCAB_SIZE);
    
    '''
    Processes train set:
    trains Backoff model and and writes the results to output file
    '''
    def procTrain(self, develFileName, firstInputWord, secondInputWord):
        
        # write number of events in development set
        self.outputMngr.writeLine(self.parser.countUniEvents(develFileName));
        
        # write number of events in validation set
        self.outputMngr.writeLine(self.parser.countUniEvents(self.validFileName));
        
        # write number of events in train set
        self.outputMngr.writeLine(self.backoff.uniLidstone.numEventsTrain);
        
        # write number of different events in train set
        self.outputMngr.writeLine(len(self.backoff.uniLidstone.obsVoc));
        
        # write frequency of first input word in train set
        self.outputMngr.writeLine(self.backoff.uniLidstone.getFreq(firstInputWord));
        
        # write frequency of the bigram (first_word second_word) in train set
        self.outputMngr.writeLine(self.backoff.biLidstone.getFreq((firstInputWord, secondInputWord)));
        
        # write perplexity of Backoff model on train set with various lambda_2 values
        self.backoff.setUniLamdba(0.1);
        
        self.backoff.setBiLamdba(0.0001);
        self.outputMngr.writeLine(self.backoff.perplexity(self.validFileName, self.trainHeader, self.wordDelim));
        
        self.backoff.setBiLamdba(0.001);
        self.outputMngr.writeLine(self.backoff.perplexity(self.validFileName, self.trainHeader, self.wordDelim));
        
        self.backoff.setBiLamdba(0.1);        
        self.outputMngr.writeLine(self.backoff.perplexity(self.validFileName, self.trainHeader, self.wordDelim));
        
        # find best lambda for Backoff's bigram Lidstone
        bestLambda = self.findBestLambda();
         
        # write best lambda
        self.outputMngr.writeLine(bestLambda);
         
        # write minimal perplexity on validation set
        self.backoff.setBiLamdba(bestLambda);
        self.outputMngr.writeLine(self.backoff.perplexity(self.validFileName, self.trainHeader, self.wordDelim));
        
    '''
    Writes to output file the perplexity of trained Backoff model on test set
    '''
    def procTest(self, testFileName):
        
        # write perplexity of trained Backoff model on test set
        self.outputMngr.writeLine(self.backoff.perplexity(testFileName, self.testHeader, self.wordDelim));
    
    '''
    Returns lambda for bigram Lidstone smoothing which minimizes
    perplexity on validation set for Backoff
    '''
    def findBestLambda(self):
        
        # initialize best lambda
        bestLambda = self.minLambda;
        self.backoff.setBiLamdba(bestLambda);
        bestPerplexity = self.backoff.perplexity(self.validFileName, self.trainHeader, self.wordDelim);
        
        # find lambda which minimizes perplexity over validation set
        numIter = int((self.maxLambda-self.minLambda) / self.step);
        for i in range(self.minLambda + 1,numIter):
            
            # calculate current lambda
            currLambda = i / 10000.0;
            self.backoff.setBiLamdba(currLambda);
            
            # calculate perplexity
            currPerplexity = self.backoff.perplexity(self.validFileName, self.trainHeader, self.wordDelim);
    
            # update best lambda
            if currPerplexity < bestPerplexity:
                bestLambda = currLambda;
                bestPerplexity = currPerplexity;
        
        # return best lambda
        return bestLambda;
    
    '''
    Writes the table to output file 
    '''
    def writeResultsTable(self, firstInputWord):
        
        # declare output
        self.outputMngr.writeLine("");
        
        # initialize observed vocabulary with probability calculated by Backoff model
        ObsVocWithProb = {}
        
        # set the new lambda of Lidstone for bigram model
        self.backoff.setBiLamdba(0.001);
        
        # build the dictionary with the observed words (in train set) and their probabilities
        for x in self.uniLidstone.obsVoc:
            ObsVocWithProb[x] = self.backoff.backoffProb((firstInputWord, x));
            
        # sort the dictionary in decreasing order of the probability value 
        myList = sorted(ObsVocWithProb.items(), key=lambda x: x[1], reverse=True);
        
        # write the results of the seen events
        for i in range(0,len(myList)):
            self.outputMngr.writeTableEntry(i+1, myList[i][0], self.biLidstone.getFreq((firstInputWord, myList[i][0])), myList[i][1]);

        # number of unseen events
        numUnseen = self.VOCAB_SIZE - len(self.uniLidstone.obsVoc);
        
        # write the results of the unseen events
        self.outputMngr.writeTableEntry(numUnseen, "UNSEEN_EVENT", 0, self.backoff.backoffProbUnseenEvent(firstInputWord));

